package com.spider.common.task;

import com.spider.common.bean.AgentVo;
import com.spider.common.utils.JedisUtils;
import com.spider.common.utils.JsonUtil;
import com.xuxueli.crawler.XxlCrawler;
import com.xuxueli.crawler.annotation.PageFieldSelect;
import com.xuxueli.crawler.annotation.PageSelect;
import com.xuxueli.crawler.conf.XxlCrawlerConf;
import com.xuxueli.crawler.model.PageRequest;
import com.xuxueli.crawler.parser.PageParser;
import com.xuxueli.crawler.util.JsoupUtil;
import com.xuxueli.crawler.util.ProxyIpUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Service;

import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.ArrayList;
import java.util.List;

/**
 * 爬虫示例07：JS渲染方式采集数据，"htmlUnit" 方案
 * (仅供学习测试使用，如有侵犯请联系删除； )
 *
 * @author xuxueli 2017-12-29 23:29:48
 */
@Component
public class AgentCrawlingTask {
    private static Logger logger = LoggerFactory.getLogger(AgentCrawlingTask.class);

    public static void main(String[] args) {
        AgentCrawlingTask agentCrawlingTask = new AgentCrawlingTask();
        agentCrawlingTask.dl();
//        GanjiGrabTaskService ganjiGrabTaskService = new GanjiGrabTaskService();
//        ganjiGrabTaskService.getHousePhone("http://sz.ganji.com/wbdetail/zufang/36439615305220x.shtml?gjcity=sz&psid=174885743202519910331775930&cookie=|||3221091388441838309967&apptype=10&entinfo=36439615305220_0&fzbref=0&key=&pubid=53538570&params=rankjxzfbestpc2099^desc&trackkey=36439615305220_07137ef7-64d6-4885-a999-36bd4400596b_20181218113825_1545104305825&fcinfotype=gz&jingxuan=1");

    }

    public void dl() {
        // 代理池
        final List<AgentVo> proxyPool = new ArrayList<AgentVo>();

        // 构造爬虫
        XxlCrawler crawler = new XxlCrawler.Builder()
                .setUrls("https://www.xicidaili.com/nn/")
                .setWhiteUrlRegexs("https://www.xicidaili.com/nn/2")      // 前2页数据
                //.setWhiteUrlRegexs(new HashSet<String>(Arrays.asList("https://www.kuaidaili.com/free/inha/\\\\d+/")))      // 全部数据
                .setThreadCount(5)
                .setPageParser(new PageParser<AgentVo>() {
                    @Override
                    public void parse(Document html, Element pageVoElement, AgentVo pageVo) {
                        if (pageVo.getPort() == 0) {
                            return;
                        }

                        Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(pageVo.getIp(), pageVo.getPort()));
                        if (ProxyIpUtil.checkProxy(proxy, "https://sz.58.com") == 200) {
                            proxyPool.add(pageVo);
                            logger.info("proxy pool size : {}, new proxy: {}", proxyPool.size(), pageVo);
                        }
                    }
                })
                .build();

        // 启动
        crawler.start(false);
        //JedisUtils.set("agents",JsonUtil.getJsonByObj(proxyPool),0);

        // 代理池数据
        logger.info("----------- proxy pool total size : {} -----------", proxyPool.size());
        logger.info(proxyPool.toString());

        // 校验代理池    (代理方式请求IP地址查询网IP138，可从页面响应确认代理是否生效)
        logger.info("----------- proxy pool check -----------");
        if (proxyPool!=null && proxyPool.size()>0) {
            for (AgentVo pageVo: proxyPool) {
                try {
                    Document html = JsoupUtil.load(new PageRequest("http://2018.ip138.com/ic.asp",
                            null,
                            null,
                            null,
                            XxlCrawlerConf.USER_AGENT_CHROME,
                            null,
                            false,
                            XxlCrawlerConf.TIMEOUT_MILLIS_DEFAULT,
                            false,
                            new Proxy(Proxy.Type.HTTP, new InetSocketAddress(pageVo.getIp(), pageVo.getPort()))));
                    logger.info(pageVo + " : " + html.html());
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }

    }

}
